package dog;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.model.ConsolePageModelPipeline;
import us.codecraft.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.processor.example.GithubRepoPageProcessor;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.text.MessageFormat;
import java.util.HashMap;
import java.util.List;

public class Magic implements PageProcessor {


//    public static final String URL_LIST = "search.51job.com\\list\\000000,000000,0000,00,9,99\\d+\\";

    public static final String URL_POST = "http://blog\\.sina\\.com\\.cn/s/blog_\\w+\\.html";
    public HashMap<String,String> urlMap=new HashMap<String, String>();
    public boolean isAdd=false;
    private Site site = Site
            .me()
            .setDomain("search.51job.com")
            .setSleepTime(300)
            .setUserAgent(
                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31");

    @Override
    public void process(Page page) {

        String str="Hello World";  //待判断的字符串
        String reg=".*search.51job.com.*";  //判断字符串中是否含有ll
        String reg2=".*jobs.51job.com.*";  //判断字符串中是否含有ll
        System.out.println(str.matches(reg));
        //列表页

//        page.putField("title", page.getHtml().xpath("//div[@id='resultList']//div[@class='el']//p//a/text()").get());
        if (page.getUrl().regex(reg).match()) {
            page.putField("content", page.getHtml().xpath("//div[@id='resultList']//div[@class='el']//p").links().all());
//            List<String> urlList=page.getHtml().xpath("//div[@class='dw_page']//ul//li").links().all();
//            System.out.println(urlList);
            String total=page.getHtml().$("input[id=hidTotalPage]","value").toString();
            System.out.println(total);
            if (isAdd==false){
                for (int i=2;i<=Integer.valueOf(total);i++)
                {
                    System.out.println("添加"+getUrl(i));

                    page.addTargetRequest(getUrl(i));
                }
                isAdd=true;
            }


//            for (String item:urlList){
////                System.out.println(item);
//                if (!urlMap.containsKey(item)){
//                    System.out.println("添加"+item);
//                }
//            }
//            page.addTargetRequests(page.getHtml().xpath("//div[@class=\"articleList\"]").links().regex(URL_POST).all());
            page.addTargetRequests(page.getHtml().xpath("//div[@id='resultList']//div[@class='el']//p").links().all());
    //文章页
            System.out.println("来了老弟");
} else {
        System.out.println("来了来了");
        System.out.println(page.getUrl());
        System.out.println(page.getHtml().xpath("//div[@class='cn']/h1/text()").toString());
        System.out.println(page.getHtml().xpath("//div[@class='cn']/strong/text()").toString());
        System.out.println(page.getHtml().xpath("//div[@class='cn']//p[@class='cname']//a/@href").toString());
        System.out.println(page.getHtml().xpath("//div[@class='cn']//p[@class='cname']//a/@title").toString());
        System.out.println(page.getHtml().xpath("//div[@class='cn']//p[@class='msg ltype']/text()").toString());
        System.out.println(page.getHtml().xpath("//div[@class='bmsg inbox']//p/text()").toString());
//            System.out.println(page.getHtml().xpath("//div[@class='tCompany_main']/text()").toString());
//            System.out.println(page.getHtml().xpath("//div[@class='tmsg inbox']/text()").toString());
//            System.out.println(page.getHtml().xpath("//div[@class='job_msg']/text()").getClass());
//            System.out.println(page.getHtml().$("div[class=bmsg job_msg inbox]").xpath("//p").all());
        List<String> list=(page.getHtml().xpath("//div[@class='bmsg job_msg inbox']//p/text()").all());
        String job_content="";
        for (String item:list){

        job_content+=item;
        }

        System.out.println(job_content);

//            System.out.println(list);
        page.putField("title", page.getHtml().xpath("//div[@class='cn']/h1/text()"));

        String finalJob_content = job_content;
        HashMap<String,String> map=new HashMap<String, String>(){
        {
        put("name", page.getHtml().xpath("//div[@class='cn']/h1/text()").toString());
        put("money", page.getHtml().xpath("//div[@class='cn']/strong/text()").toString());
        put("url",page.getUrl().toString());
        put("company_name",page.getHtml().xpath("//div[@class='cn']//p[@class='cname']//a/@title").toString());
        put("company_url",page.getHtml().xpath("//div[@class='cn']//p[@class='cname']//a/@href").toString());
        put("city",page.getHtml().xpath("//div[@class='cn']//p[@class='msg ltype']/text()").toString());
        put("content", finalJob_content);
        put("address", page.getHtml().xpath("//div[@class='bmsg inbox']//p/text()").toString());
        put("search_key", "php");
        }
        };
        try {
            Thread.sleep(1000);
            insertData(map);
        }catch (Exception ex){
            System.out.println(ex.getMessage());
        }

//            page.putField("content", page.getHtml().xpath("//div[@id='articlebody']//div[@class='articalContent']"));
//            page.putField("date",
//                    page.getHtml().xpath("//div[@id='articlebody']//span[@class='time SG_txtc']").regex("\\((.*)\\)"));
        }
    }

    public String   driver = "com.mysql.jdbc.Driver";

    // URL指向要访问的数据库名scutcs
    public String url = "jdbc:mysql://top.wlphp.cn:3305/demo_cai_top";

    // MySQL配置时的用户名
    public String user = "root";

    // MySQL配置时的密码
    public String password = "123456";

    public void insertData(HashMap<String,String> map){
        try {
            Class.forName(driver);

            // 连续数据库
            Connection conn = DriverManager.getConnection(url, user, password);

            if(!conn.isClosed())
                System.out.println("Succeeded connecting to the Database!");

            // statement用来执行SQL语句
            Statement statement = conn.createStatement();

            // 要执行的SQL语句
            String sql = "INSERT INTO jobs(name,url,company_name,company_url,city,content,address,money,search_key)VALUES('1','2','3','4','5','6','7','8','9')";

            sql= MessageFormat.format("INSERT INTO jobs(name,url,company_name,company_url,city,content,address,money,search_key)VALUES(\"{0}\",\"{1}\",\"{2}\",\"{3}\",\"{4}\",\"{5}\",\"{6}\",\"{7}\",\"{8}\")"
                    ,map.get("name"),
                    map.get("url"),map.get("company_name"),map.get("company_url"),map.get("city"),map.get("content"),
                    map.get("address"),map.get("money"),map.get("search_key"));



            System.out.println(sql);
            // 结果集
            boolean rs = statement.execute(sql);

            System.out.println(rs);

        }catch (Exception ex){

        }

    }


    @Override
    public Site getSite() {
        return site;
    }

    public static String getUrl(int page){
        return "https://search.51job.com/list/000000,000000,0000,00,9,99,php,2,"+page+".html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=";

    }

    public static void main(String[] args) {
        Spider.create(new Magic())
                .addUrl(getUrl(1))
                .addPipeline(new JsonFilePipeline("/Users/mac/www/java/code"))
//                .addPipeline(new ConsolePageModelPipeline())
                //开启5个线程抓取
                .thread(1)

                .run();
    }
}